{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "排名前十的电影为：\n",
      "[4.234 average rating (583 reviews)] Star Wars (1977)\n",
      "[4.224 average rating (298 reviews)] Schindler's List (1993)\n",
      "[4.196 average rating (283 reviews)] Shawshank Redemption, The (1994)\n",
      "[4.172 average rating (243 reviews)] Casablanca (1942)\n",
      "[4.135 average rating (267 reviews)] Usual Suspects, The (1995)\n",
      "[4.123 average rating (413 reviews)] Godfather, The (1972)\n",
      "[4.120 average rating (390 reviews)] Silence of the Lambs, The (1991)\n",
      "[4.098 average rating (420 reviews)] Raiders of the Lost Ark (1981)\n",
      "[4.082 average rating (209 reviews)] Rear Window (1954)\n",
      "[4.066 average rating (350 reviews)] Titanic (1997)\n",
      "用户643和用户916的偏好相似性为： 0.08563786063744523\n",
      "用户643和用户916的相关性为： 0.3942560673329039\n",
      "用户766相似（皮尔逊系数）的影评人为：\n",
      "  15: 1.000\n",
      "  32: 1.000\n",
      "  39: 1.000\n",
      "  40: 1.000\n",
      "  46: 1.000\n",
      "  54: 1.000\n",
      "  68: 1.000\n",
      "  74: 1.000\n",
      " 112: 1.000\n",
      " 133: 1.000\n",
      "用户766相似（欧氏距离）的影评人为：\n",
      "  46: 1.000\n",
      "  61: 1.000\n",
      "  79: 1.000\n",
      " 101: 1.000\n",
      " 105: 1.000\n",
      " 129: 1.000\n",
      " 139: 1.000\n",
      " 157: 1.000\n",
      " 181: 1.000\n",
      " 191: 1.000\n",
      "用户422对电影50的评分（欧氏距离）为： 4.354131517221993\n",
      "用户422对电影50的评分（皮尔逊系数）为： 4.3566797825971575\n",
      "用户532对所有电影评分中排名前十的是：\n",
      "5.000: Santa with Muscles (1996)\n",
      "5.000: Great Day in Harlem, A (1994)\n",
      "5.000: They Made Me a Criminal (1939)\n",
      "5.000: Prefontaine (1997)\n",
      "5.000: Marlene Dietrich: Shadow and Light (1996) \n",
      "5.000: Star Kid (1997)\n",
      "5.000: Saint of Fort Washington, The (1993)\n",
      "5.000: Aiqing wansui (1994)\n",
      "5.000: Everest (1998)\n",
      "5.000: Someone Else's America (1995)\n"
     ]
    }
   ],
   "source": [
    "# encoding:utf-8\n",
    "\n",
    "import os\n",
    "import csv\n",
    "import heapq\n",
    "import math\n",
    "from operator import itemgetter\n",
    "from datetime import datetime\n",
    "from collections import defaultdict\n",
    "\n",
    "# 加载电影数据文件（包含用户ID、电影ID、评分、时间戳）\n",
    "def load_reviews(path, **kwargs):\n",
    "    options = {\n",
    "        'fieldnames': ('userid', 'movieid', 'rating', 'timestamp'),\n",
    "        'delimiter': '\\t'\n",
    "    }\n",
    "    options.update(kwargs)\n",
    "    parse_date = lambda r, k: datetime.fromtimestamp(float(r[k]))\n",
    "    parse_int = lambda r, k: int(r[k])\n",
    "    with open(path, 'rt', encoding=\"utf-8\") as reviews:\n",
    "        reader = csv.DictReader(reviews, **options)\n",
    "        for row in reader:\n",
    "            row['movieid'] = parse_int(row, 'movieid')\n",
    "            row['userid'] = parse_int(row, 'userid')\n",
    "            row['rating'] = parse_int(row, 'rating')\n",
    "            row['timestamp'] = parse_date(row, 'timestamp')\n",
    "            yield row\n",
    "\n",
    "# 辅助数据导入\n",
    "def relative_path(path):\n",
    "    dirname = os.path.dirname(os.path.realpath('__file__'))\n",
    "    path = os.path.join(dirname, path)\n",
    "    return  os.path.normpath(path)\n",
    "\n",
    "# 读取电影信息（包含电影ID、电影名、出品时间、视频地址、类型涵盖）\n",
    "def load_movies(path, **kwargs):\n",
    "    options = {\n",
    "        'fieldnames': ('movieid', 'title', 'release', 'video', 'url'),\n",
    "        'delimiter': '|',\n",
    "        'restkey': 'genre'\n",
    "    }\n",
    "    options.update(**kwargs)\n",
    "    parse_int = lambda r, k: int(r[k])\n",
    "    parse_date = lambda r, k: datetime.strptime(r[k], '%d-%b-%Y') if r[k] else None\n",
    "    with open(path, 'rt', encoding=\"utf-8\") as movies:\n",
    "        reader = csv.DictReader(movies, **options)\n",
    "        for row in reader:\n",
    "            row['movieid'] = parse_int(row, 'movieid')\n",
    "            # print row['movieid']\n",
    "            row['release'] = parse_date(row, 'release')\n",
    "            # print row['release']\n",
    "            # print row['video']\n",
    "            yield row\n",
    "\n",
    "# 基于用户的推荐系统:\n",
    "class MovieLens(object):\n",
    "\n",
    "    def __init__(self, udata, uitem):\n",
    "        self.udata = udata\n",
    "        self.uitem = uitem\n",
    "        self.movies = {}\n",
    "        self.reviews = defaultdict(dict)\n",
    "        self.load_dataset()\n",
    "\n",
    "    def load_dataset(self):\n",
    "        # 加载数据到内存中，按ID为索引\n",
    "        for movie in load_movies(self.uitem):\n",
    "            self.movies[movie['movieid']] = movie\n",
    "\n",
    "        for review in load_reviews(self.udata):\n",
    "            self.reviews[review['userid']][review['movieid']] = review\n",
    "            # print self.reviews[review['userid']][review['movieid']]\n",
    "\n",
    "    # 检查电影是否存在\n",
    "    def reviews_for_movie(self, movieid):\n",
    "        for review in self.reviews.values():\n",
    "            if movieid in review:\n",
    "                yield review[movieid]\n",
    "\n",
    "    # 对所有的电影求平均评价分数\n",
    "    def average_reviews(self):\n",
    "        for movieid in self.movies:\n",
    "            reviews = list(r['rating'] for r in self.reviews_for_movie(movieid))\n",
    "            average = sum(reviews) / float(len(reviews))\n",
    "            yield (movieid, average, len(reviews))  # 返回了（movieid，评分平均分，长度(即评价人数)）\n",
    "\n",
    "    # 获取电影前n的top排行\n",
    "    def top_rated(self, n=10):\n",
    "        return heapq.nlargest(n, self.bayesian_average(), key=itemgetter(1))\n",
    "\n",
    "    # 获取一个修正后的贝叶斯平均值\n",
    "    def bayesian_average(self, c=59, m=3):\n",
    "        for movieid in self.movies:\n",
    "            reviews = list(r['rating'] for r in self.reviews_for_movie(movieid))\n",
    "            average = ((c * m) + sum(reviews)) / float(c + len(reviews))\n",
    "            yield (movieid, average, len(reviews))\n",
    "\n",
    "    # 找出两个评论者之间的交集\n",
    "    def share_preferences(self, criticA, criticB):\n",
    "        if criticA not in self.reviews:\n",
    "            raise KeyError(\"Couldn't find critic '%s' in data \" % criticA)\n",
    "        if criticB not in self.reviews:\n",
    "            raise KeyError(\"Couldn't find critic '%s' in data \" % criticB)\n",
    "        moviesA = set(self.reviews[criticA].keys())\n",
    "        moviesB = set(self.reviews[criticB].keys())\n",
    "        shared = moviesA & moviesB\n",
    "        # 创建一个评论过的的字典返回\n",
    "        reviews = {}\n",
    "        for movieid in shared:\n",
    "            reviews[movieid] = (\n",
    "                self.reviews[criticA][movieid]['rating'],\n",
    "                self.reviews[criticB][movieid]['rating'],\n",
    "            )\n",
    "        return reviews\n",
    "\n",
    "    # 通过两个人的共同偏好作为向量来计算两个用户之间的欧式距离\n",
    "    def euclidean_distance(self, criticA, criticB, prefs='users'):\n",
    "        # 创建两个用户的交集\n",
    "        preferences = self.share_preferences(criticA, criticB)\n",
    "        # 没有则返回0\n",
    "        if len(preferences) == 0: return 0\n",
    "        # 求偏差的平方的和\n",
    "        sum_of_squares = sum([pow(a - b, 2) for a, b in preferences.values()])\n",
    "        # 修正的欧式距离，返回值的范围为[0,1]\n",
    "        return 1 / (1 + math.sqrt(sum_of_squares))\n",
    "\n",
    "    # 返回两个评论者之间的皮尔逊相关系数\n",
    "    def pearson_correlation(self, criticA, criticB, prefs='users'):\n",
    "        if prefs == 'users':\n",
    "            preferences = self.share_preferences(criticA, criticB)\n",
    "        elif prefs == 'movies':\n",
    "            preferences = self.shared_critics(criticA, criticB)\n",
    "        else:\n",
    "            raise Exception(\"No preferences of type '%s'.\" % prefs)\n",
    "        length = len(preferences)\n",
    "        if length == 0: return 0\n",
    "        # 循环处理每一个评论者之间的皮尔逊相关系数\n",
    "        sumA = sumB = sumSquareA = sumSquareB = sumProducts = 0\n",
    "        for a, b in preferences.values():\n",
    "            sumA += a\n",
    "            sumB += b\n",
    "            sumSquareA += pow(a, 2)\n",
    "            sumSquareB += pow(b, 2)\n",
    "            sumProducts += a * b\n",
    "        # 计算皮尔逊系数\n",
    "        numerator = (sumProducts * length) - (sumA * sumB)\n",
    "        denominator = math.sqrt(((sumSquareA * length) - pow(sumA, 2)) * ((sumSquareB * length) - pow(sumB, 2)))\n",
    "        if denominator == 0: return 0\n",
    "        return abs(numerator / denominator)\n",
    "\n",
    "    # 为特定用户寻找一个合适的影评人\n",
    "    def similar_critics(self, user, metric='euclidean', n=None):\n",
    "        metrics = {\n",
    "            'euclidean': self.euclidean_distance,\n",
    "            'pearson': self.pearson_correlation\n",
    "        }\n",
    "        distance = metrics.get(metric, None)\n",
    "        # 解决可能出现的状况\n",
    "        if user not in self.reviews:\n",
    "            raise KeyError(\"Unknown user, '%s'.\" % user)\n",
    "        if not distance or not callable(distance):\n",
    "            raise KeyError(\"Unknown or unprogrammed distance metric '%s'.\" % metric)\n",
    "        # 计算对用户最合适的影评人\n",
    "        critics = {}\n",
    "        for critic in self.reviews:\n",
    "            # 不能与自己进行比较\n",
    "            if critic == user:\n",
    "                continue\n",
    "            critics[critic] = distance(user, critic)\n",
    "        if n:\n",
    "            return heapq.nlargest(n, critics.items(), key=itemgetter(1))\n",
    "        return critics\n",
    "\n",
    "    '''\n",
    "    预测一个用户对一部电影的评分，相当于评论过这部电影的用户对当前用户的加权均值\n",
    "    并且权重取决与其他用户和该用户的相似程度\n",
    "    '''\n",
    "    def predict_ranking(self, user, movie, metric='euclidean', critics=None):\n",
    "        critics = critics or self.similar_critics(user, metric=metric)\n",
    "        total = 0.0\n",
    "        simsum = 0.0\n",
    "        for critic, similarity in critics.items():\n",
    "            if movie in self.reviews[critic]:\n",
    "                total += similarity * self.reviews[critic][movie]['rating']\n",
    "                simsum += similarity\n",
    "        if simsum == 0.0: return 0.0\n",
    "        return total / simsum\n",
    "\n",
    "    # 为所有的电影预测评分，返回前n个评分的电影和它们的评分\n",
    "    def predict_all_rankings(self, user, metric='euclidean', n=None):\n",
    "        critics = self.similar_critics(user, metric=metric)\n",
    "        movies = {\n",
    "            movie: self.predict_ranking(user, movie, metric, critics)\n",
    "            for movie in self.movies\n",
    "        }\n",
    "        if n:\n",
    "            return heapq.nlargest(n, movies.items(), key=itemgetter(1))\n",
    "        return movies\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    data = relative_path('data/ml-100k/u.data')\n",
    "    item = relative_path('data/ml-100k/u.item')\n",
    "    model = MovieLens(data, item)\n",
    "    # 基于用户的相关预测\n",
    "    # 输出排名前十的电影\n",
    "    print (\"排名前十的电影为：\")\n",
    "    for mid, avg, num in model.top_rated(10):\n",
    "        title = model.movies[mid]['title']\n",
    "        print (\"[%0.3f average rating (%i reviews)] %s\" % (avg, num,title))\n",
    "    # 计算两个用户在偏好空间中的距离\n",
    "    print (\"用户643和用户916的偏好相似性为：\", model.euclidean_distance(643, 916))  # A,B\n",
    "    # 计算两个用户相关性\n",
    "    print (\"用户643和用户916的相关性为：\",model.pearson_correlation(643, 916))\n",
    "    # 利用皮尔逊系数为特定用户寻找最好的影评人\n",
    "    print (\"用户766相似（皮尔逊系数）的影评人为：\")\n",
    "    for item in model.similar_critics(766, 'pearson', n=10):\n",
    "        print (\"%4i: %0.3f\" % item)\n",
    "    # 利用欧氏距离为特定用户寻找最好的影评人\n",
    "    print(\"用户766相似（欧氏距离）的影评人为：\")\n",
    "    for item in model.similar_critics(766, 'euclidean', n=10):\n",
    "        print (\"%4i: %0.3f\" % item)\n",
    "    # 预测一个用户对一部电影的评分\n",
    "    print (\"用户422对电影50的评分（欧氏距离）为：\",model.predict_ranking(422, 50, 'euclidean'))\n",
    "    print (\"用户422对电影50的评分（皮尔逊系数）为：\",model.predict_ranking(422, 50, 'pearson'))\n",
    "    # 预测一个用户对所有电影的评分，并返回前n个值\n",
    "    print (\"用户532对所有电影评分中排名前十的是：\")\n",
    "    for mid ,rating in model.predict_all_rankings(532,'pearson',10):\n",
    "        print ('%0.3f: %s' % (rating, model.movies[mid]['title']))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
